package io.sqrtqiezi.spark.join

import org.apache.spark.rdd.RDD
import org.apache.spark.{HashPartitioner, RangePartitioner, SparkConf, SparkContext}

object JoinDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setAppName(this.getClass.getCanonicalName.init)
      .setMaster("local[*]")
    val sc = new SparkContext(conf)

    val random = scala.util.Random

    val col1 = Range(1, 10).map(idx => (random.nextInt(10), s"user$idx"))
    val col2 = Array(
      (0, "BJ"), (1, "SH"), (2, "GZ"), (3, "SZ"), (4, "TJ"),
      (5, "CQ"), (6, "HZ"), (7, "NJ"), (8, "WH"), (9, "CD"))
    val rdd1: RDD[(Int, String)] = sc.makeRDD(col1)
    val rdd2: RDD[(Int, String)] = sc.makeRDD(col2)

    rdd1.collect.foreach(println)

    println(s"RDD1 partitions number: ${rdd1.getNumPartitions}")
    println(rdd1.partitioner)

//    val rdd3 = rdd1.join(rdd2)
//    println(rdd3.dependencies)
//
//    val rdd4 = rdd1.partitionBy(new HashPartitioner(3))
//      .join(rdd2)
//    println(rdd4.dependencies)
//
//    val rdd5 = rdd1.groupByKey()
//    println(rdd5.dependencies)

    sc.stop()
  }
}
